

<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>rl_coach.agents.soft_actor_critic_agent &mdash; Reinforcement Learning Coach 0.12.0 documentation</title>
  

  
  
  
  

  
  <script type="text/javascript" src="../../../_static/js/modernizr.min.js"></script>
  
    
      <script type="text/javascript" id="documentation_options" data-url_root="../../../" src="../../../_static/documentation_options.js"></script>
        <script type="text/javascript" src="../../../_static/jquery.js"></script>
        <script type="text/javascript" src="../../../_static/underscore.js"></script>
        <script type="text/javascript" src="../../../_static/doctools.js"></script>
        <script type="text/javascript" src="../../../_static/language_data.js"></script>
        <script async="async" type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/latest.js?config=TeX-AMS-MML_HTMLorMML"></script>
    
    <script type="text/javascript" src="../../../_static/js/theme.js"></script>

    

  
  <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../../../_static/pygments.css" type="text/css" />
  <link rel="stylesheet" href="../../../_static/css/custom.css" type="text/css" />
    <link rel="index" title="Index" href="../../../genindex.html" />
    <link rel="search" title="Search" href="../../../search.html" />
    <link href="../../../_static/css/custom.css" rel="stylesheet" type="text/css">

</head>

<body class="wy-body-for-nav">

   
  <div class="wy-grid-for-nav">
    
    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
      <div class="wy-side-scroll">
        <div class="wy-side-nav-search" >
          

          
            <a href="../../../index.html" class="icon icon-home"> Reinforcement Learning Coach
          

          
            
            <img src="../../../_static/dark_logo.png" class="logo" alt="Logo"/>
          
          </a>

          
            
            
          

          
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

          
        </div>

        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          
            
            
              
            
            
              <p class="caption"><span class="caption-text">Intro</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../usage.html">Usage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dist_usage.html">Usage - Distributed Coach</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../features/index.html">Features</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../selecting_an_algorithm.html">Selecting an Algorithm</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dashboard.html">Coach Dashboard</a></li>
</ul>
<p class="caption"><span class="caption-text">Design</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../design/control_flow.html">Control Flow</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../design/network.html">Network Design</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../design/horizontal_scaling.html">Distributed Coach - Horizontal Scale-Out</a></li>
</ul>
<p class="caption"><span class="caption-text">Contributing</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../contributing/add_agent.html">Adding a New Agent</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../contributing/add_env.html">Adding a New Environment</a></li>
</ul>
<p class="caption"><span class="caption-text">Components</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../components/agents/index.html">Agents</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../components/architectures/index.html">Architectures</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../components/data_stores/index.html">Data Stores</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../components/environments/index.html">Environments</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../components/exploration_policies/index.html">Exploration Policies</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../components/filters/index.html">Filters</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../components/memories/index.html">Memories</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../components/memory_backends/index.html">Memory Backends</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../components/orchestrators/index.html">Orchestrators</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../components/core_types.html">Core Types</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../components/spaces.html">Spaces</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../components/additional_parameters.html">Additional Parameters</a></li>
</ul>

            
          
        </div>
      </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">

      
      <nav class="wy-nav-top" aria-label="top navigation">
        
          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
          <a href="../../../index.html">Reinforcement Learning Coach</a>
        
      </nav>


      <div class="wy-nav-content">
        
        <div class="rst-content">
        
          















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="wy-breadcrumbs">
    
      <li><a href="../../../index.html">Docs</a> &raquo;</li>
        
          <li><a href="../../index.html">Module code</a> &raquo;</li>
        
      <li>rl_coach.agents.soft_actor_critic_agent</li>
    
    
      <li class="wy-breadcrumbs-aside">
        
      </li>
    
  </ul>

  
  <hr/>
</div>
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
  <h1>Source code for rl_coach.agents.soft_actor_critic_agent</h1><div class="highlight"><pre>
<span></span><span class="c1">#</span>
<span class="c1"># Copyright (c) 2019 Intel Corporation</span>
<span class="c1">#</span>
<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
<span class="c1"># you may not use this file except in compliance with the License.</span>
<span class="c1"># You may obtain a copy of the License at</span>
<span class="c1">#</span>
<span class="c1">#      http://www.apache.org/licenses/LICENSE-2.0</span>
<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
<span class="c1"># See the License for the specific language governing permissions and</span>
<span class="c1"># limitations under the License.</span>
<span class="c1">#</span>

<span class="kn">from</span> <span class="nn">typing</span> <span class="k">import</span> <span class="n">Union</span>
<span class="kn">import</span> <span class="nn">copy</span>
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
<span class="kn">from</span> <span class="nn">collections</span> <span class="k">import</span> <span class="n">OrderedDict</span>

<span class="kn">from</span> <span class="nn">rl_coach.agents.agent</span> <span class="k">import</span> <span class="n">Agent</span>
<span class="kn">from</span> <span class="nn">rl_coach.agents.policy_optimization_agent</span> <span class="k">import</span> <span class="n">PolicyOptimizationAgent</span>

<span class="kn">from</span> <span class="nn">rl_coach.architectures.head_parameters</span> <span class="k">import</span> <span class="n">SACQHeadParameters</span><span class="p">,</span><span class="n">SACPolicyHeadParameters</span><span class="p">,</span><span class="n">VHeadParameters</span>
<span class="kn">from</span> <span class="nn">rl_coach.architectures.middleware_parameters</span> <span class="k">import</span> <span class="n">FCMiddlewareParameters</span>
<span class="kn">from</span> <span class="nn">rl_coach.base_parameters</span> <span class="k">import</span> <span class="n">AlgorithmParameters</span><span class="p">,</span> <span class="n">NetworkParameters</span><span class="p">,</span> <span class="n">AgentParameters</span><span class="p">,</span> <span class="n">EmbedderScheme</span><span class="p">,</span> <span class="n">MiddlewareScheme</span>
<span class="kn">from</span> <span class="nn">rl_coach.core_types</span> <span class="k">import</span> <span class="n">ActionInfo</span><span class="p">,</span> <span class="n">EnvironmentSteps</span><span class="p">,</span> <span class="n">RunPhase</span>
<span class="kn">from</span> <span class="nn">rl_coach.exploration_policies.additive_noise</span> <span class="k">import</span> <span class="n">AdditiveNoiseParameters</span>
<span class="kn">from</span> <span class="nn">rl_coach.memories.non_episodic.experience_replay</span> <span class="k">import</span> <span class="n">ExperienceReplayParameters</span>
<span class="kn">from</span> <span class="nn">rl_coach.architectures.embedder_parameters</span> <span class="k">import</span> <span class="n">InputEmbedderParameters</span>
<span class="kn">from</span> <span class="nn">rl_coach.spaces</span> <span class="k">import</span> <span class="n">BoxActionSpace</span>


<span class="c1"># There are 3 networks in SAC implementation. All have the same topology but parameters are not shared.</span>
<span class="c1"># The networks are:</span>
<span class="c1"># 1. State Value Network - SACValueNetwork</span>
<span class="c1"># 2. Soft Q Value Network - SACCriticNetwork</span>
<span class="c1"># 3. Policy Network - SACPolicyNetwork - currently supporting only Gaussian Policy</span>


<span class="c1"># 1. State Value Network - SACValueNetwork</span>
<span class="c1"># this is the state value network in SAC.</span>
<span class="c1"># The network is trained to predict (regression) the state value in the max-entropy settings</span>
<span class="c1"># The objective to be minimized is given in equation (5) in the paper:</span>
<span class="c1">#</span>
<span class="c1"># J(psi)= E_(s~D)[0.5*(V_psi(s)-y(s))^2]</span>
<span class="c1"># where y(s) = E_(a~pi)[Q_theta(s,a)-log(pi(a|s))]</span>


<span class="c1"># Default parameters for value network:</span>
<span class="c1"># topology :</span>
<span class="c1">#   input embedder : EmbedderScheme.Medium (Dense(256)) , relu activation</span>
<span class="c1">#   middleware : EmbedderScheme.Medium (Dense(256)) , relu activation</span>


<span class="k">class</span> <span class="nc">SACValueNetworkParameters</span><span class="p">(</span><span class="n">NetworkParameters</span><span class="p">):</span>
    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">input_embedders_parameters</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;observation&#39;</span><span class="p">:</span> <span class="n">InputEmbedderParameters</span><span class="p">(</span><span class="n">activation_function</span><span class="o">=</span><span class="s1">&#39;relu&#39;</span><span class="p">)}</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">middleware_parameters</span> <span class="o">=</span> <span class="n">FCMiddlewareParameters</span><span class="p">(</span><span class="n">activation_function</span><span class="o">=</span><span class="s1">&#39;relu&#39;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">heads_parameters</span> <span class="o">=</span> <span class="p">[</span><span class="n">VHeadParameters</span><span class="p">(</span><span class="n">initializer</span><span class="o">=</span><span class="s1">&#39;xavier&#39;</span><span class="p">)]</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">rescale_gradient_from_head_by_factor</span> <span class="o">=</span> <span class="p">[</span><span class="mi">1</span><span class="p">]</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">optimizer_type</span> <span class="o">=</span> <span class="s1">&#39;Adam&#39;</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">batch_size</span> <span class="o">=</span> <span class="mi">256</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">async_training</span> <span class="o">=</span> <span class="kc">False</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">learning_rate</span> <span class="o">=</span> <span class="mf">0.0003</span>     <span class="c1"># 3e-4 see appendix D in the paper</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">create_target_network</span> <span class="o">=</span> <span class="kc">True</span>   <span class="c1"># tau is set in SoftActorCriticAlgorithmParameters.rate_for_copying_weights_to_target</span>


<span class="c1"># 2. Soft Q Value Network - SACCriticNetwork</span>
<span class="c1"># the whole network is built in the SACQHeadParameters. we use empty input embedder and middleware</span>
<span class="k">class</span> <span class="nc">SACCriticNetworkParameters</span><span class="p">(</span><span class="n">NetworkParameters</span><span class="p">):</span>
    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">input_embedders_parameters</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;observation&#39;</span><span class="p">:</span> <span class="n">InputEmbedderParameters</span><span class="p">(</span><span class="n">scheme</span><span class="o">=</span><span class="n">EmbedderScheme</span><span class="o">.</span><span class="n">Empty</span><span class="p">)}</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">middleware_parameters</span> <span class="o">=</span> <span class="n">FCMiddlewareParameters</span><span class="p">(</span><span class="n">scheme</span><span class="o">=</span><span class="n">MiddlewareScheme</span><span class="o">.</span><span class="n">Empty</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">heads_parameters</span> <span class="o">=</span> <span class="p">[</span><span class="n">SACQHeadParameters</span><span class="p">()]</span>      <span class="c1"># SACQHeadParameters includes the topology of the head</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">rescale_gradient_from_head_by_factor</span> <span class="o">=</span> <span class="p">[</span><span class="mi">1</span><span class="p">]</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">optimizer_type</span> <span class="o">=</span> <span class="s1">&#39;Adam&#39;</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">batch_size</span> <span class="o">=</span> <span class="mi">256</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">async_training</span> <span class="o">=</span> <span class="kc">False</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">learning_rate</span> <span class="o">=</span> <span class="mf">0.0003</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">create_target_network</span> <span class="o">=</span> <span class="kc">False</span>


<span class="c1"># 3. policy Network</span>
<span class="c1"># Default parameters for policy network:</span>
<span class="c1"># topology :</span>
<span class="c1">#   input embedder : EmbedderScheme.Medium (Dense(256)) , relu activation</span>
<span class="c1">#   middleware : EmbedderScheme = [Dense(256)] , relu activation --&gt; scheme should be overridden in preset</span>
<span class="k">class</span> <span class="nc">SACPolicyNetworkParameters</span><span class="p">(</span><span class="n">NetworkParameters</span><span class="p">):</span>
    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">input_embedders_parameters</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;observation&#39;</span><span class="p">:</span> <span class="n">InputEmbedderParameters</span><span class="p">(</span><span class="n">activation_function</span><span class="o">=</span><span class="s1">&#39;relu&#39;</span><span class="p">)}</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">middleware_parameters</span> <span class="o">=</span> <span class="n">FCMiddlewareParameters</span><span class="p">(</span><span class="n">activation_function</span><span class="o">=</span><span class="s1">&#39;relu&#39;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">heads_parameters</span> <span class="o">=</span> <span class="p">[</span><span class="n">SACPolicyHeadParameters</span><span class="p">()]</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">rescale_gradient_from_head_by_factor</span> <span class="o">=</span> <span class="p">[</span><span class="mi">1</span><span class="p">]</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">optimizer_type</span> <span class="o">=</span> <span class="s1">&#39;Adam&#39;</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">batch_size</span> <span class="o">=</span> <span class="mi">256</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">async_training</span> <span class="o">=</span> <span class="kc">False</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">learning_rate</span> <span class="o">=</span> <span class="mf">0.0003</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">create_target_network</span> <span class="o">=</span> <span class="kc">False</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">l2_regularization</span> <span class="o">=</span> <span class="mi">0</span>      <span class="c1"># weight decay regularization. not used in the original paper</span>


<span class="c1"># Algorithm Parameters</span>

<div class="viewcode-block" id="SoftActorCriticAlgorithmParameters"><a class="viewcode-back" href="../../../components/agents/policy_optimization/sac.html#rl_coach.agents.soft_actor_critic_agent.SoftActorCriticAlgorithmParameters">[docs]</a><span class="k">class</span> <span class="nc">SoftActorCriticAlgorithmParameters</span><span class="p">(</span><span class="n">AlgorithmParameters</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    :param num_steps_between_copying_online_weights_to_target: (StepMethod)</span>
<span class="sd">        The number of steps between copying the online network weights to the target network weights.</span>

<span class="sd">    :param rate_for_copying_weights_to_target: (float)</span>
<span class="sd">        When copying the online network weights to the target network weights, a soft update will be used, which</span>
<span class="sd">        weight the new online network weights by rate_for_copying_weights_to_target. (Tau as defined in the paper)</span>

<span class="sd">    :param use_deterministic_for_evaluation: (bool)</span>
<span class="sd">        If True, during the evaluation phase, action are chosen deterministically according to the policy mean</span>
<span class="sd">        and not sampled from the policy distribution.</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">num_steps_between_copying_online_weights_to_target</span> <span class="o">=</span> <span class="n">EnvironmentSteps</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">rate_for_copying_weights_to_target</span> <span class="o">=</span> <span class="mf">0.005</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">use_deterministic_for_evaluation</span> <span class="o">=</span> <span class="kc">True</span>    <span class="c1"># evaluate agent using deterministic policy (i.e. take the mean value)</span></div>


<span class="k">class</span> <span class="nc">SoftActorCriticAgentParameters</span><span class="p">(</span><span class="n">AgentParameters</span><span class="p">):</span>
    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">algorithm</span><span class="o">=</span><span class="n">SoftActorCriticAlgorithmParameters</span><span class="p">(),</span>
                         <span class="n">exploration</span><span class="o">=</span><span class="n">AdditiveNoiseParameters</span><span class="p">(),</span>
                         <span class="n">memory</span><span class="o">=</span><span class="n">ExperienceReplayParameters</span><span class="p">(),</span>   <span class="c1"># SAC doesnt use episodic related data</span>
                         <span class="c1"># network wrappers:</span>
                         <span class="n">networks</span><span class="o">=</span><span class="n">OrderedDict</span><span class="p">([(</span><span class="s2">&quot;policy&quot;</span><span class="p">,</span> <span class="n">SACPolicyNetworkParameters</span><span class="p">()),</span>
                                               <span class="p">(</span><span class="s2">&quot;q&quot;</span><span class="p">,</span> <span class="n">SACCriticNetworkParameters</span><span class="p">()),</span>
                                               <span class="p">(</span><span class="s2">&quot;v&quot;</span><span class="p">,</span> <span class="n">SACValueNetworkParameters</span><span class="p">())]))</span>

    <span class="nd">@property</span>
    <span class="k">def</span> <span class="nf">path</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">return</span> <span class="s1">&#39;rl_coach.agents.soft_actor_critic_agent:SoftActorCriticAgent&#39;</span>


<span class="c1"># Soft Actor Critic - https://arxiv.org/abs/1801.01290</span>
<span class="k">class</span> <span class="nc">SoftActorCriticAgent</span><span class="p">(</span><span class="n">PolicyOptimizationAgent</span><span class="p">):</span>
    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">agent_parameters</span><span class="p">,</span> <span class="n">parent</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="s1">&#39;LevelManager&#39;</span><span class="p">,</span> <span class="s1">&#39;CompositeAgent&#39;</span><span class="p">]</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">agent_parameters</span><span class="p">,</span> <span class="n">parent</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">last_gradient_update_step_idx</span> <span class="o">=</span> <span class="mi">0</span>

        <span class="c1"># register signals to track (in learn_from_batch)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">policy_means</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">register_signal</span><span class="p">(</span><span class="s1">&#39;Policy_mu_avg&#39;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">policy_logsig</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">register_signal</span><span class="p">(</span><span class="s1">&#39;Policy_logsig&#39;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">policy_logprob_sampled</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">register_signal</span><span class="p">(</span><span class="s1">&#39;Policy_logp_sampled&#39;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">policy_grads</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">register_signal</span><span class="p">(</span><span class="s1">&#39;Policy_grads_sumabs&#39;</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">q1_values</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">register_signal</span><span class="p">(</span><span class="s2">&quot;Q1&quot;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">TD_err1</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">register_signal</span><span class="p">(</span><span class="s2">&quot;TD err1&quot;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">q2_values</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">register_signal</span><span class="p">(</span><span class="s2">&quot;Q2&quot;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">TD_err2</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">register_signal</span><span class="p">(</span><span class="s2">&quot;TD err2&quot;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">v_tgt_ns</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">register_signal</span><span class="p">(</span><span class="s1">&#39;V_tgt_ns&#39;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">v_onl_ys</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">register_signal</span><span class="p">(</span><span class="s1">&#39;V_onl_ys&#39;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">action_signal</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">register_signal</span><span class="p">(</span><span class="s2">&quot;actions&quot;</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">learn_from_batch</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">batch</span><span class="p">):</span>
        <span class="c1">#########################################</span>
        <span class="c1"># need to update the following networks:</span>
        <span class="c1"># 1. actor (policy)</span>
        <span class="c1"># 2. state value (v)</span>
        <span class="c1"># 3. critic (q1 and q2)</span>
        <span class="c1"># 4. target network - probably already handled by V</span>

        <span class="c1">#########################################</span>
        <span class="c1"># define the networks to be used</span>

        <span class="c1"># State Value Network</span>
        <span class="n">value_network</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">networks</span><span class="p">[</span><span class="s1">&#39;v&#39;</span><span class="p">]</span>
        <span class="n">value_network_keys</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">ap</span><span class="o">.</span><span class="n">network_wrappers</span><span class="p">[</span><span class="s1">&#39;v&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">input_embedders_parameters</span><span class="o">.</span><span class="n">keys</span><span class="p">()</span>

        <span class="c1"># Critic Network</span>
        <span class="n">q_network</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">networks</span><span class="p">[</span><span class="s1">&#39;q&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">online_network</span>
        <span class="n">q_head</span> <span class="o">=</span> <span class="n">q_network</span><span class="o">.</span><span class="n">output_heads</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
        <span class="n">q_network_keys</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">ap</span><span class="o">.</span><span class="n">network_wrappers</span><span class="p">[</span><span class="s1">&#39;q&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">input_embedders_parameters</span><span class="o">.</span><span class="n">keys</span><span class="p">()</span>

        <span class="c1"># Actor (policy) Network</span>
        <span class="n">policy_network</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">networks</span><span class="p">[</span><span class="s1">&#39;policy&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">online_network</span>
        <span class="n">policy_network_keys</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">ap</span><span class="o">.</span><span class="n">network_wrappers</span><span class="p">[</span><span class="s1">&#39;policy&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">input_embedders_parameters</span><span class="o">.</span><span class="n">keys</span><span class="p">()</span>

        <span class="c1">##########################################</span>
        <span class="c1"># 1. updating the actor - according to (13) in the paper</span>
        <span class="n">policy_inputs</span> <span class="o">=</span> <span class="n">copy</span><span class="o">.</span><span class="n">copy</span><span class="p">(</span><span class="n">batch</span><span class="o">.</span><span class="n">states</span><span class="p">(</span><span class="n">policy_network_keys</span><span class="p">))</span>
        <span class="n">policy_results</span> <span class="o">=</span> <span class="n">policy_network</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">policy_inputs</span><span class="p">)</span>

        <span class="n">policy_mu</span><span class="p">,</span> <span class="n">policy_std</span><span class="p">,</span> <span class="n">sampled_raw_actions</span><span class="p">,</span> <span class="n">sampled_actions</span><span class="p">,</span> <span class="n">sampled_actions_logprob</span><span class="p">,</span> \
        <span class="n">sampled_actions_logprob_mean</span> <span class="o">=</span> <span class="n">policy_results</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">policy_means</span><span class="o">.</span><span class="n">add_sample</span><span class="p">(</span><span class="n">policy_mu</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">policy_logsig</span><span class="o">.</span><span class="n">add_sample</span><span class="p">(</span><span class="n">policy_std</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">policy_logprob_sampled</span><span class="o">.</span><span class="n">add_sample</span><span class="p">(</span><span class="n">sampled_actions_logprob_mean</span><span class="p">)</span>

        <span class="c1"># get the state-action values for the replayed states and their corresponding actions from the policy</span>
        <span class="n">q_inputs</span> <span class="o">=</span> <span class="n">copy</span><span class="o">.</span><span class="n">copy</span><span class="p">(</span><span class="n">batch</span><span class="o">.</span><span class="n">states</span><span class="p">(</span><span class="n">q_network_keys</span><span class="p">))</span>
        <span class="n">q_inputs</span><span class="p">[</span><span class="s1">&#39;output_0_0&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">sampled_actions</span>
        <span class="n">log_target</span> <span class="o">=</span> <span class="n">q_network</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">q_inputs</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">squeeze</span><span class="p">()</span>

        <span class="c1"># log internal q values</span>
        <span class="n">q1_vals</span><span class="p">,</span> <span class="n">q2_vals</span> <span class="o">=</span> <span class="n">q_network</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">q_inputs</span><span class="p">,</span> <span class="n">outputs</span><span class="o">=</span><span class="p">[</span><span class="n">q_head</span><span class="o">.</span><span class="n">q1_output</span><span class="p">,</span> <span class="n">q_head</span><span class="o">.</span><span class="n">q2_output</span><span class="p">])</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">q1_values</span><span class="o">.</span><span class="n">add_sample</span><span class="p">(</span><span class="n">q1_vals</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">q2_values</span><span class="o">.</span><span class="n">add_sample</span><span class="p">(</span><span class="n">q2_vals</span><span class="p">)</span>

        <span class="c1"># calculate the gradients according to (13)</span>
        <span class="c1"># get the gradients of log_prob w.r.t the weights (parameters) - indicated as phi in the paper</span>
        <span class="n">initial_feed_dict</span> <span class="o">=</span> <span class="p">{</span><span class="n">policy_network</span><span class="o">.</span><span class="n">gradients_weights_ph</span><span class="p">[</span><span class="mi">5</span><span class="p">]:</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="mf">1.0</span><span class="p">)}</span>
        <span class="n">dlogp_dphi</span> <span class="o">=</span> <span class="n">policy_network</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">policy_inputs</span><span class="p">,</span>
                                            <span class="n">outputs</span><span class="o">=</span><span class="n">policy_network</span><span class="o">.</span><span class="n">weighted_gradients</span><span class="p">[</span><span class="mi">5</span><span class="p">],</span>
                                            <span class="n">initial_feed_dict</span><span class="o">=</span><span class="n">initial_feed_dict</span><span class="p">)</span>

        <span class="c1"># calculate dq_da</span>
        <span class="n">dq_da</span> <span class="o">=</span> <span class="n">q_network</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">q_inputs</span><span class="p">,</span>
                                  <span class="n">outputs</span><span class="o">=</span><span class="n">q_network</span><span class="o">.</span><span class="n">gradients_wrt_inputs</span><span class="p">[</span><span class="mi">1</span><span class="p">][</span><span class="s1">&#39;output_0_0&#39;</span><span class="p">])</span>

        <span class="c1"># calculate da_dphi</span>
        <span class="n">initial_feed_dict</span> <span class="o">=</span> <span class="p">{</span><span class="n">policy_network</span><span class="o">.</span><span class="n">gradients_weights_ph</span><span class="p">[</span><span class="mi">3</span><span class="p">]:</span> <span class="n">dq_da</span><span class="p">}</span>
        <span class="n">dq_dphi</span> <span class="o">=</span> <span class="n">policy_network</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">policy_inputs</span><span class="p">,</span>
                                         <span class="n">outputs</span><span class="o">=</span><span class="n">policy_network</span><span class="o">.</span><span class="n">weighted_gradients</span><span class="p">[</span><span class="mi">3</span><span class="p">],</span>
                                         <span class="n">initial_feed_dict</span><span class="o">=</span><span class="n">initial_feed_dict</span><span class="p">)</span>

        <span class="c1"># now given dlogp_dphi, dq_dphi we need to calculate the policy gradients according to (13)</span>
        <span class="n">policy_grads</span> <span class="o">=</span> <span class="p">[</span><span class="n">dlogp_dphi</span><span class="p">[</span><span class="n">l</span><span class="p">]</span> <span class="o">-</span> <span class="n">dq_dphi</span><span class="p">[</span><span class="n">l</span><span class="p">]</span> <span class="k">for</span> <span class="n">l</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">dlogp_dphi</span><span class="p">))]</span>

        <span class="c1"># apply the gradients to policy networks</span>
        <span class="n">policy_network</span><span class="o">.</span><span class="n">apply_gradients</span><span class="p">(</span><span class="n">policy_grads</span><span class="p">)</span>
        <span class="n">grads_sumabs</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">([</span><span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">abs</span><span class="p">(</span><span class="n">policy_grads</span><span class="p">[</span><span class="n">l</span><span class="p">]))</span> <span class="k">for</span> <span class="n">l</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">policy_grads</span><span class="p">))])</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">policy_grads</span><span class="o">.</span><span class="n">add_sample</span><span class="p">(</span><span class="n">grads_sumabs</span><span class="p">)</span>

        <span class="c1">##########################################</span>
        <span class="c1"># 2. updating the state value online network weights</span>
        <span class="c1"># done by calculating the targets for the v head according to (5) in the paper</span>
        <span class="c1"># value_targets = log_targets-sampled_actions_logprob</span>
        <span class="n">value_inputs</span> <span class="o">=</span> <span class="n">copy</span><span class="o">.</span><span class="n">copy</span><span class="p">(</span><span class="n">batch</span><span class="o">.</span><span class="n">states</span><span class="p">(</span><span class="n">value_network_keys</span><span class="p">))</span>
        <span class="n">value_targets</span> <span class="o">=</span> <span class="n">log_target</span> <span class="o">-</span> <span class="n">sampled_actions_logprob</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">v_onl_ys</span><span class="o">.</span><span class="n">add_sample</span><span class="p">(</span><span class="n">value_targets</span><span class="p">)</span>

        <span class="c1"># call value_network apply gradients with this target</span>
        <span class="n">value_loss</span> <span class="o">=</span> <span class="n">value_network</span><span class="o">.</span><span class="n">online_network</span><span class="o">.</span><span class="n">train_on_batch</span><span class="p">(</span><span class="n">value_inputs</span><span class="p">,</span> <span class="n">value_targets</span><span class="p">[:,</span><span class="kc">None</span><span class="p">])[</span><span class="mi">0</span><span class="p">]</span>

        <span class="c1">##########################################</span>
        <span class="c1"># 3. updating the critic (q networks)</span>
        <span class="c1"># updating q networks according to (7) in the paper</span>

        <span class="c1"># define the input to the q network: state has been already updated previously, but now we need</span>
        <span class="c1"># the actions from the batch (and not those sampled by the policy)</span>
        <span class="n">q_inputs</span><span class="p">[</span><span class="s1">&#39;output_0_0&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">batch</span><span class="o">.</span><span class="n">actions</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">batch</span><span class="o">.</span><span class="n">actions</span><span class="p">()</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span><span class="p">)</span>

        <span class="c1"># define the targets : scale_reward * reward + (1-terminal)*discount*v_target_next_state</span>
        <span class="c1"># define v_target_next_state</span>
        <span class="n">value_inputs</span> <span class="o">=</span> <span class="n">copy</span><span class="o">.</span><span class="n">copy</span><span class="p">(</span><span class="n">batch</span><span class="o">.</span><span class="n">next_states</span><span class="p">(</span><span class="n">value_network_keys</span><span class="p">))</span>
        <span class="n">v_target_next_state</span> <span class="o">=</span> <span class="n">value_network</span><span class="o">.</span><span class="n">target_network</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">value_inputs</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">v_tgt_ns</span><span class="o">.</span><span class="n">add_sample</span><span class="p">(</span><span class="n">v_target_next_state</span><span class="p">)</span>
        <span class="c1"># Note: reward is assumed to be rescaled by RewardRescaleFilter in the preset parameters</span>
        <span class="n">TD_targets</span> <span class="o">=</span> <span class="n">batch</span><span class="o">.</span><span class="n">rewards</span><span class="p">(</span><span class="n">expand_dims</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> <span class="o">+</span> \
                     <span class="p">(</span><span class="mf">1.0</span> <span class="o">-</span> <span class="n">batch</span><span class="o">.</span><span class="n">game_overs</span><span class="p">(</span><span class="n">expand_dims</span><span class="o">=</span><span class="kc">True</span><span class="p">))</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">ap</span><span class="o">.</span><span class="n">algorithm</span><span class="o">.</span><span class="n">discount</span> <span class="o">*</span> <span class="n">v_target_next_state</span>

        <span class="c1"># call critic network update</span>
        <span class="n">result</span> <span class="o">=</span> <span class="n">q_network</span><span class="o">.</span><span class="n">train_on_batch</span><span class="p">(</span><span class="n">q_inputs</span><span class="p">,</span> <span class="n">TD_targets</span><span class="p">,</span> <span class="n">additional_fetches</span><span class="o">=</span><span class="p">[</span><span class="n">q_head</span><span class="o">.</span><span class="n">q1_loss</span><span class="p">,</span> <span class="n">q_head</span><span class="o">.</span><span class="n">q2_loss</span><span class="p">])</span>
        <span class="n">total_loss</span><span class="p">,</span> <span class="n">losses</span><span class="p">,</span> <span class="n">unclipped_grads</span> <span class="o">=</span> <span class="n">result</span><span class="p">[:</span><span class="mi">3</span><span class="p">]</span>
        <span class="n">q1_loss</span><span class="p">,</span> <span class="n">q2_loss</span> <span class="o">=</span> <span class="n">result</span><span class="p">[</span><span class="mi">3</span><span class="p">]</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">TD_err1</span><span class="o">.</span><span class="n">add_sample</span><span class="p">(</span><span class="n">q1_loss</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">TD_err2</span><span class="o">.</span><span class="n">add_sample</span><span class="p">(</span><span class="n">q2_loss</span><span class="p">)</span>

        <span class="c1">##########################################</span>
        <span class="c1"># 4. updating the value target network</span>
        <span class="c1"># I just need to set the parameter rate_for_copying_weights_to_target in the agent parameters to be 1-tau</span>
        <span class="c1"># where tau is the hyper parameter as defined in sac original implementation</span>

        <span class="k">return</span> <span class="n">total_loss</span><span class="p">,</span> <span class="n">losses</span><span class="p">,</span> <span class="n">unclipped_grads</span>

    <span class="k">def</span> <span class="nf">get_prediction</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">states</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        get the mean and stdev of the policy distribution given &#39;states&#39;</span>
<span class="sd">        :param states: the states for which we need to sample actions from the policy</span>
<span class="sd">        :return: mean and stdev</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">tf_input_state</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">prepare_batch_for_inference</span><span class="p">(</span><span class="n">states</span><span class="p">,</span> <span class="s1">&#39;policy&#39;</span><span class="p">)</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">networks</span><span class="p">[</span><span class="s1">&#39;policy&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">online_network</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">tf_input_state</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">train</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="c1"># since the algorithm works with experience replay buffer (non-episodic),</span>
        <span class="c1"># we cant use the policy optimization train method. we need Agent.train</span>
        <span class="c1"># note that since in Agent.train there is no apply_gradients, we need to do it in learn from batch</span>
        <span class="k">return</span> <span class="n">Agent</span><span class="o">.</span><span class="n">train</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">choose_action</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">curr_state</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        choose_action - chooses the most likely action</span>
<span class="sd">        if &#39;deterministic&#39; - take the mean of the policy which is the prediction of the policy network.</span>
<span class="sd">        else - use the exploration policy</span>
<span class="sd">        :param curr_state:</span>
<span class="sd">        :return: action wrapped in ActionInfo</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">spaces</span><span class="o">.</span><span class="n">action</span><span class="p">,</span> <span class="n">BoxActionSpace</span><span class="p">):</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;SAC works only for continuous control problems&quot;</span><span class="p">)</span>
        <span class="c1"># convert to batch so we can run it through the network</span>
        <span class="n">tf_input_state</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">prepare_batch_for_inference</span><span class="p">(</span><span class="n">curr_state</span><span class="p">,</span> <span class="s1">&#39;policy&#39;</span><span class="p">)</span>
        <span class="c1"># use the online network for prediction</span>
        <span class="n">policy_network</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">networks</span><span class="p">[</span><span class="s1">&#39;policy&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">online_network</span>
        <span class="n">policy_head</span> <span class="o">=</span> <span class="n">policy_network</span><span class="o">.</span><span class="n">output_heads</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
        <span class="n">result</span> <span class="o">=</span> <span class="n">policy_network</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">tf_input_state</span><span class="p">,</span>
                                        <span class="n">outputs</span><span class="o">=</span><span class="p">[</span><span class="n">policy_head</span><span class="o">.</span><span class="n">policy_mean</span><span class="p">,</span> <span class="n">policy_head</span><span class="o">.</span><span class="n">actions</span><span class="p">])</span>
        <span class="n">action_mean</span><span class="p">,</span> <span class="n">action_sample</span> <span class="o">=</span> <span class="n">result</span>

        <span class="c1"># if using deterministic policy, take the mean values. else, use exploration policy to sample from the pdf</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">phase</span> <span class="o">==</span> <span class="n">RunPhase</span><span class="o">.</span><span class="n">TEST</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">ap</span><span class="o">.</span><span class="n">algorithm</span><span class="o">.</span><span class="n">use_deterministic_for_evaluation</span><span class="p">:</span>
            <span class="n">action</span> <span class="o">=</span> <span class="n">action_mean</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">action</span> <span class="o">=</span> <span class="n">action_sample</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">action_signal</span><span class="o">.</span><span class="n">add_sample</span><span class="p">(</span><span class="n">action</span><span class="p">)</span>

        <span class="n">action_info</span> <span class="o">=</span> <span class="n">ActionInfo</span><span class="p">(</span><span class="n">action</span><span class="o">=</span><span class="n">action</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">action_info</span>
</pre></div>

           </div>
           
          </div>
          <footer>
  

  <hr/>

  <div role="contentinfo">
    <p>
        &copy; Copyright 2018-2019, Intel AI Lab

    </p>
  </div>
  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 

</footer>

        </div>
      </div>

    </section>

  </div>
  


  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>

  
  
    
   

</body>
</html>